package it.java.giw.dataextraction.aleph.crawler;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Lock;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.eclipse.jetty.util.ConcurrentHashSet;

import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.WebResponse;
import com.gargoylesoftware.htmlunit.html.DomElement;
import com.gargoylesoftware.htmlunit.html.DomNodeList;
import com.gargoylesoftware.htmlunit.html.HtmlPage;

public class CrawlerWorker implements Runnable{

	private LinkedBlockingDeque<String> seedList;
	private LinkedBlockingQueue<String> workInProgress;
	private ConcurrentHashSet<Integer> alreadyVisited;
	private final CountDownLatch startSignal;
	private final CountDownLatch doneSignal;
	private AtomicInteger id;
	private Lock lock;
	private List<String> newSeedsFound;
	private ArrayList<String> url_patterns;
	private final int LIMIT = 100;

	private static final String hostPattern = "(?:http|https)://(\\w+([\\.|-]\\w+)*\\.[a-z]+)";

	public CrawlerWorker(LinkedBlockingDeque<String> seedList, LinkedBlockingQueue<String> workInProgress, Lock lock,
			ConcurrentHashSet<Integer> alreadyVisited, AtomicInteger id, CountDownLatch startSignal, CountDownLatch doneSignal) {
		this.seedList = seedList;
		this.workInProgress = workInProgress;
		this.alreadyVisited = alreadyVisited;
		this.startSignal = startSignal;
		this.doneSignal = doneSignal;
		this.id = id;
		this.lock = lock;
		this.url_patterns = new ArrayList<String>();
		
		url_patterns.add("http://www.venere.com/it/hotel/.*");
//		url_patterns.add("http://www.lafeltrinelli.it/libri/\\w+[-]?\\w+/\\w+[-]?\\w+/[0-9]+$");
		
//		url_patterns.add("http://www.mymovies.it/dizionario/recensione.asp\\?id=.*$");
//		url_patterns.add("http://www.cinemadelsilenzio.it/index.php\\?mod=film&id=[0-9]*");
//		url_patterns.add("http://www.comingsoon.it//film/.*/scheda/$");
//		url_patterns.add("http://www.film.it/films/o/.*");
//		url_patterns.add("http://www.bestmovie.it/film-trailer/.*/$");
	}

	/**
	 * Questo metodo prende una stringa e un pattern, e controlla
	 * se la stringa matcha il pattern.
	 * In caso positivo restituisce il pezzo di stringa che ha matchato.
	 * In caso contrario restituisce la stringa vuota.
	 * @param url
	 * @param pattern
	 * @return
	 */
	private static String parse(String url, String pattern) {
		Pattern patt = Pattern.compile(pattern);
		Matcher mat = patt.matcher(url);
		String result = "";
		if(mat.find())
			result = mat.group(1);
		return result;
	}

	@Override
	public void run() {
		/* create webClient and set options to get only HTML */
		final WebClient webClient = new WebClient();
//		webClient.getOptions().setThrowExceptionOnScriptError(false);
		webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
		webClient.getOptions().setCssEnabled(false);
		webClient.getOptions().setJavaScriptEnabled(false);
		webClient.getOptions().setAppletEnabled(false);

		try{
			startSignal.await();
			String url;
			while(id.get() < LIMIT){
				HtmlPage page = null;
				lock.lock();
				try{
					if((seedList.isEmpty() && workInProgress.isEmpty())){
						break;
					}
					url = seedList.pollFirst();
					if(url!=null)
						workInProgress.add(url);
				}finally{
					lock.unlock();
				}
				/* caso in cui seedList e' vuota e workInProgress no */
				if (url != null){
					System.out.println("Sono il thread: "+ Thread.currentThread().getId());
					try{
						alreadyVisited.add(url.hashCode());
						System.out.println("Analyzing URL: " + url);
						page = webClient.getPage(url);
						WebResponse response = page.getWebResponse();
						String content = response.getContentAsString();
						if (matchPattern(url, url_patterns)){
							id.incrementAndGet();
							File file = new File("download/"+id+".html");
							// if file doesnt exists, then create it
							if (!file.exists()) {
								file.createNewFile();
							}
							FileWriter fw = new FileWriter(file.getAbsoluteFile());
							BufferedWriter bw = new BufferedWriter(fw);
							bw.write(content);
							bw.close();
							
							long threadId = Thread.currentThread().getId();
							File index = new File("id2url"+threadId+".txt");
							// if file doesnt exists, then create it
							if (!index.exists()) {
								index.createNewFile();
							}
							fw = new FileWriter(index.getAbsoluteFile(), true);
							bw = new BufferedWriter(fw);
							bw.write(id.toString()+"-->"+page.getUrl().toString()+"\n");
							bw.close();

							
						}
						DomNodeList<DomElement> linkElements = page.getElementsByTagName("a");
						newSeedsFound = new ArrayList<String>();
						for(DomElement e : linkElements){
							String attr_value = e.getAttribute("href");
							String source = parse(attr_value, hostPattern);

							attr_value = attr_value.replaceAll("#.*", "");
							attr_value = attr_value.replaceAll("mailto:.*", "");
							if (!"".equals(attr_value)){
								if(source.isEmpty()){
									String url_string = "";
									if(attr_value.startsWith("/"))
										url_string = "http://"+page.getUrl().getHost()+ attr_value;
									else
										url_string = "http://"+page.getUrl().getHost()+ "/" + attr_value;
									if(alreadyVisited.add(url_string.hashCode())){
										newSeedsFound.add(url_string);
									}
								}
								else{
									/* analizza solo gli url che sono del dominio di interesse */
									if(page.getUrl().getHost().equals(source)){
										String url_string = attr_value;
										if(alreadyVisited.add(url_string.hashCode())){
											newSeedsFound.add(url_string);
										}
									}
								}
							}
						}			

					}catch(Exception e){
						e.printStackTrace();
					}finally{
						lock.lock();
						try{							
							for (String s : newSeedsFound)
								seedList.addLast(s);
							workInProgress.remove(url);
						}finally{
							lock.unlock();
						}
					}

				}
			}
			doneSignal.countDown();
		}catch(Exception e){
			e.printStackTrace();
			doneSignal.countDown();
		}
	}

	private boolean matchPattern(String attr_value,	ArrayList<String> url_patterns2) {
		for (String pattern : url_patterns2 ){
			if (attr_value.matches(pattern))
				return true;
		}
		return false;
	}
}
